import requests
from n2 import HnswIndex
import time
import threading
import numpy as np
import os
import redis
import pymysql
from smtplib import SMTP_SSL
from email.header import Header
from email.mime.text import MIMEText


# 邮箱配置信息
def warning_email():
	mail_info = {
		"from": "pbgs04@163.com",
		"to": "43861@126.com",
		"hostname": "smtp.163.com",
		"username": "pbgs04@163.com",
		"password": "aabbcc123456",
		"mail_subject": "vcgEditorial_n2_warning",
		"mail_text": "warning, this is a port 237 errorlog email, sended by vcgEditorial_n2",
		"mail_encoding": "utf-8"
	}
	# 这里使用SMTP_SSL就是默认使用465端口，25测试有防火墙
	smtp = SMTP_SSL(mail_info["hostname"])
	# smtp.set_debuglevel(1)
	smtp.ehlo(mail_info["hostname"])
	smtp.login(mail_info["username"], mail_info["password"])
	msg = MIMEText(mail_info["mail_text"], "plain", mail_info["mail_encoding"])
	msg["Subject"] = Header(mail_info["mail_subject"], mail_info["mail_encoding"])
	msg["from"] = mail_info["from"]
	msg["to"] = mail_info["to"]
	smtp.sendmail(mail_info["from"], mail_info["to"], msg.as_string())
	smtp.quit()


# 数据库日志模块
def log_mysql(start_id, end_id, un_match_num, start_time, end_time, use_time, asset_family=1):
	db = pymysql.connect("rm-2zezq40w18v8dqcy2.mysql.rds.aliyuncs.com", "root", "2bWP*qU8W$%", "zpt")
	db1 = pymysql.connect("rm-2zezq40w18v8dqcy2.mysql.rds.aliyuncs.com", "root", "2bWP*qU8W$%", "new_add")
	cursor = db.cursor()
	cursor1 = db1.cursor()
	sql1 = """SELECT COUNT(*) from ele_index_editorial WHERE id BETWEEN %s and %s""" % (start_id, end_id)
	# print(sql1)
	cursor1.execute(sql1)
	num_data = cursor1.fetchone()
	db1.close()
	update_num = num_data[0]
	# print(update_num)
	# SQL 插入语句
	sql = """INSERT INTO pic_update_time_log(start_id ,end_id,update_num,un_match_num,start_time,end_time,use_time,asset_family)
		         VALUES (%s, %s, %s,%s,'%s','%s', %s, %s)""" % (
		start_id, end_id, update_num, un_match_num, start_time, end_time, use_time, asset_family)
	# print(sql)
	try:
		# 执行sql语句
		cursor.execute(sql)
		# 提交到数据库执行
		db.commit()
	except:
		# 如果发生错误则回滚
		db.rollback()

	# 关闭数据库连接
	db.close()


def getallfile(path):
	allfile = []
	allfilelist = os.listdir(path)
	for file in allfilelist:
		filepath = os.path.join(path, file)
		allfile.append(filepath)
	return allfile


def getFilename(filename):
	(filepath, tempfilename) = os.path.split(filename)
	(shotname, extension) = os.path.splitext(tempfilename)
	return shotname


def getFileExtension(filename):
	(filepath, tempfilename) = os.path.split(filename)
	(shotname, extension) = os.path.splitext(tempfilename)
	return extension


# 删除上一次n2文件
def delete_last_n2():
	cmd = "rm - f /data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/editvcg*M.n2"
	os.system(cmd)


# 删除上一次sucesslog
def delete_last_sucesslog():
	cmd = "rm - f /data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/sucesslogtmp_*"
	os.system(cmd)


# 删除上一次errorlog
def delete_last_errorlog():
	cmd = "rm - f /data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/errorlogtmp_*"
	os.system(cmd)


# 编辑类
def copy_to_csv_ready():
	cmd = "cp /data1/auto_add/tmp_1_1_*.csv /data1/auto_add/vcgEditorial_data/vcgEditorial_csv_ready"
	# print(cmd)
	os.system(cmd)


# 删除csv文件
def delete_from_csv_ready():
	cmd = "rm -f /data1/auto_add/vcgEditorial_data/vcgEditorial_csv_ready/*"
	os.system(cmd)


# 删除检查文件
def delete_error_log_txt():
	cmd = "rm - f /data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/*_1_1_errorlog.txt"
	os.system(cmd)


def get_true_filename(filename):
	tmplist = filename.split('_')
	# print(tmplist)
	# exit()
	sort = int(tmplist[1])
	asset_family = int(tmplist[2])
	csvno = str(tmplist[3])

	if (sort == 1 or sort == 2) and asset_family == 2:
		n2_prefix_name = 'vcgCreativeIndex'
	elif (sort == 1 or sort == 2) and asset_family == 1:
		n2_prefix_name = 'editvcg'
	elif sort == 3:
		n2_prefix_name = 'veerlog'
	elif sort == 4:
		n2_prefix_name = '500pxlog'
	elif sort == 5:
		n2_prefix_name = 'worldcup2018'
		tmp_af = 5
	elif sort == 6:
		n2_prefix_name = 'worldcupALL'
		tmp_af = 6

	true_csv_name = n2_prefix_name + csvno + 'M.n2'
	return true_csv_name


def read_successlog(filename, ):
	tmplist = filename.split('_')
	sort = tmplist[1]
	asset_family = tmplist[2]
	csvno = tmplist[3]


def open_db():
	global db, cursor
	db = pymysql.connect("rm-2zezq40w18v8dqcy2.mysql.rds.aliyuncs.com", "root", "2bWP*qU8W$%", "new_add")
	cursor = db.cursor()


def close_db():
	global db
	db.close()


def open_db_ele_index_edit():
	global db_index_edit, cursor_index_edit
	db_index_edit = pymysql.connect("rm-2zezq40w18v8dqcy2.mysql.rds.aliyuncs.com", "root", "2bWP*qU8W$%", "new_add")
	cursor_index_edit = db_index_edit.cursor()


def close_db_index_edit():
	global db_index_edit
	db_index_edit.close()


def build_n2():
	print('step 1 done:' + str(time.time()))
	# 第一步：将每日追加的csv临时文件，拷贝到csv_ready目录：#########################################
	copy_to_csv_ready()
	print('step 2 done:' + str(time.time()))

	# 第二步：开始跑csv_ready目录下的csv文件######################################################
	myPath = '/data1/auto_add/vcgEditorial_data/vcgEditorial_csv_ready/'
	allfiles = getallfile(myPath)
	print(allfiles)
	for item in allfiles:
		f = 2048  # Weight
		t = HnswIndex(f)  # HnswIndex(f, "L2 or angular")
		print(item)
		filename = str(getFilename(item))
		print(filename)
		sucesslogName = 'sucesslog' + filename
		errorlogName = 'errorlog' + filename
		with open(item, 'r') as fi, open(errorlogName, 'w') as fo, open(sucesslogName, 'w') as fo1:
			for i in range(2000000):
				chunk_data = fi.readline()
				if not chunk_data:
					print('chunk_data is done')
					break
				myID = chunk_data.split('\t')[0]
				strVector = chunk_data.split('\t')[1]
				features_arr = strVector.split(',')
				features_arr = [a for a in features_arr if len(a) > 0]
				features_arr = np.array(features_arr, dtype=np.float64)

				try:
					t.add_data(features_arr)
					successInfo = myID + '\n'
					fo1.write(successInfo)
				except:
					errInfo = myID + '|' + strVector + '\n'
					fo.write(errInfo)

		t.build(m=1000, n_threads=32)
		taget_index_name = filename + '.n2'
		t.save(taget_index_name)
	print('step 3 done:' + str(time.time()))
	# exit()

	# 第三步：循环全部successlog文件 写入redis######################################################
	open_db()
	myPath = '/data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/'
	allsuccessfiles = getallfile(myPath)
	r = redis.Redis(host='r-2zea3c0ff18d2e94.redis.rds.aliyuncs.com', port=6379, db=0, password='b3bPVU8W',
	                decode_responses=True)

	for item in allsuccessfiles:
		filename = str(getFilename(item))

		if 'sucesslog' in filename:
			tmplist = filename.split('_')
			sort = int(tmplist[1])

			if sort == 1:
				asset_family = int(tmplist[2])
			elif sort == 3:
				asset_family = 3
			elif sort == 4:
				asset_family = 4

			csvno = int(tmplist[3])
			fd = open('/data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/' + filename)
			# print('/data1/auto_add/n2_ready/'+filename)
			# exit()
			index_no = csvno / 2
			index_id = 0
			for line in fd:
				if index_id % 10000 == 0:
					print(index_id)
				result_id = line.strip()
				r_key = '%d_%d_%d' % (index_no, index_id, asset_family)
				if r.exists(r_key) != True:
					r.set(r_key, result_id)
					update_ele_index(sort, asset_family, index_no, index_id, result_id)
				index_id += 1

	close_db()
	print('step 4 done:' + str(time.time()))

	# 第四步：将n2临时文件名 改为 在线文件名################################################
	myPath = '/data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/'
	allcsvfiles = getallfile(myPath)
	print(allcsvfiles)

	for item in allcsvfiles:
		ext = str(getFileExtension(item))
		filename = str(getFilename(item))

		if ext == '.n2':
			true_n2_name = get_true_filename(filename)
			cmd = "mv %s /data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/%s" % (item, true_n2_name)
			print(cmd)
			os.system(cmd)

	print('step 5 done:' + str(time.time()))

	# 第五步：删除csv_ready目录下的csv文件##################################################
	delete_from_csv_ready()

	print('step 6 done:' + str(time.time()))


auto_edit_n2_web_url = 'http://172.16.241.109:8080/view/%E5%A4%A7%E8%B1%A1%E7%9B%B8%E4%BC%BC%E5%9B%BE%E8%87%AA%E5%8A%A8%E5%8C%96%E5%8F%91%E5%B8%83/job/auto_edit_n2_update/build?token=auto_edit_n2_deploy'
auto_edit_n2_tencent_url ='http://172.16.241.109:8080/view/%E8%85%BE%E8%AE%AF%E7%89%88%E6%9D%83%E6%9B%B4%E6%96%B0/job/txbq_auto_edit_n2_update/build?token=txbq_auto_edit_n2_deploy'

# 连接jenkins
def connect_genkins_edit(url):
	# 重定向以后的登录验证url
	headers = {
		'user-agent': "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36"
	}
	login_url = 'http://172.16.241.109:8080/j_acegi_security_check'
	data = {
		'j_username': 'wenxiangpeng',
		'j_password': 'qw*&Bao2'
	}
	# 建立会话
	s = requests.session()
	r = s.post(url=login_url, data=data, headers=headers)

	# 任务url
	# auto_edit_n2_deploy_url = 'http://172.16.241.109:8080/view/%E5%A4%A7%E8%B1%A1%E7%9B%B8%E4%BC%BC%E5%9B%BE%E8%87%AA%E5%8A%A8%E5%8C%96%E5%8F%91%E5%B8%83/job/auto_edit_n2_update/build?token=auto_edit_n2_deploy'
	# headers = {
	# 	'Referer': 'http://39.106.177.73:8080/login?from=%2F'
	# }
	response = s.get(url=url, headers=headers)
	print(response.status_code)


# 写入消息队列，参数1：队列名称，参数2：消息内容
def update_ele_index(sort, asset_family, index_no, index_id, result_id):
	global db, cursor

	if (sort == 1 or sort == 2) and asset_family == 2:
		tbname = 'new_add.ele_index'
		tmp_af = 2
	elif (sort == 1 or sort == 2) and asset_family == 1:
		tbname = 'new_add.ele_index_editorial'
		tmp_af = 1
	elif sort == 3:
		tbname = 'new_add.ele_index_veer'
		tmp_af = 3
	elif sort == 4:
		tbname = 'new_add.ele_index_500px'
		tmp_af = 4
	elif sort == 5:
		tbname = 'new_add.ele_index_worldcup2018'
		tmp_af = 5
	elif sort == 6:
		tbname = 'new_add.ele_index_worldcupALL'
		tmp_af = 6
	# print(type(asset_family))
	# exit()
	sql = """update %s set index_no=%d, index_id=%d, asset_family=%d where result_id='%s'""" % (
		tbname, index_no, index_id, tmp_af, result_id)
	# print(sql)
	# exit()
	try:
		# 执行sql语句
		cursor.execute(sql)
		# 提交到数据库执行
		db.commit()
	except:
		# 如果发生错误则回滚
		print(sql)
		db.rollback()
		return False

	return True


###################    主要流程      #########################

flag_tencent = 'vcg_editor_tencent'
flag_web = 'vcg_editor_web'

r = redis.Redis(host='r-2zea3c0ff18d2e94.redis.rds.aliyuncs.com', port=6379, db=0, password='b3bPVU8W',
                decode_responses=True)

# 启动时默认定为 0    0 启动 or 等待   1  开始生成 N2   2 开始
r.set(flag_tencent, 0)
r.set(flag_web, 0)

while True:
	flag_tencent_state = int(r.get(flag_tencent))
	flag_web_state = int(r.get(flag_web))
	print('flag_tencent_state: ', flag_tencent_state)
	print('flag_web_state：', flag_web_state)

	if (flag_tencent_state == 0 and flag_web_state == 1) or (flag_tencent_state == 1 and flag_web_state == 0) or (
			flag_tencent_state == 0 and flag_web_state == 0):
		time.sleep(60)
		continue

	elif flag_tencent_state == 1 and flag_web_state == 1:
		r.set('tx_vcgCreative_state',0)
		r.set('web_vcgCreative_state',0)
		# 检查是否有新数据写入
		open_db_ele_index_edit()
		sql = """select id from ele_index_editorial order by id DESC limit 1"""
		# print(sql)
		cursor_index_edit.execute(sql)
		current_last_id_data = cursor_index_edit.fetchone()
		current_last_id = current_last_id_data[0]
		print(current_last_id)

		close_db_index_edit()
		vcgEditorial_last_last_id_num = r.get('vcgEditorial_last_last_id')
		vcgEditorial_last_last_id = int(vcgEditorial_last_last_id_num)
		print(vcgEditorial_last_last_id)
		if current_last_id == vcgEditorial_last_last_id:
			time.sleep(300)
			continue

		print('开始生成n2')
		t1 = time.time()  # 开始时间
		start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
		start_id = vcgEditorial_last_last_id
		end_id = current_last_id
		# 清空上次结果
		delete_last_n2()
		delete_last_sucesslog()
		delete_last_errorlog()
		delete_error_log_txt()

		#主函数
		build_n2()

		# 开始处理
		os.system('python  /data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/f2.py')

		# 获取目录下所有文件，返回一个列表
		my_errorlog_path = '/data1/auto_add/vcgEditorial_data/vcgEditorial_n2_ready/'
		allcsvfiles_2 = getallfile(my_errorlog_path)
		print(allcsvfiles_2)
		txt_list = []

		# 获取目录下所有txt文件
		for item in allcsvfiles_2:
			# print(item)
			ext = str(getFileExtension(item))
			# print(ext)
			filename = str(getFilename(item))
			# print(filename)

			if ext == '.txt':
				filenames = filename + '.txt'
				txt_list.append(filenames)
		print(txt_list)
		count = 0
		error_log_num = []
		for i in txt_list:
			print(i)

			fp = open(i, "r", encoding='utf-8')
			while 1:
				buffer = fp.read(8 * 1024 * 1024)
				if not buffer:
					break
				count += buffer.count('\n')

			print(count)
			error_log_num.append(count)
			fp.close()

		print('over')
		if len(error_log_num) != 0:
			un_match_num = max(error_log_num)
			if max(error_log_num) > 30:
				print('数据异常不能更新')
				warning_email()
				break
		else:
			un_match_num = 0

		# 上传

		th1 = threading.Thread(target=connect_genkins_edit, args=(auto_edit_n2_web_url,))  # 生成一个线程实

		th2 = threading.Thread(target=connect_genkins_edit, args=(auto_edit_n2_tencent_url,))  # 生成另一个线程实例

		th1.start()
		th2.start()
		print(th1.getName())
		print(th2.getName())
		th1.join()
		th2.join()

		t2 = time.time()  # 结束时间
		end_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
		use_time = t2 - t1  # 使用时间
		log_mysql(start_id, end_id, un_match_num, start_time, end_time, use_time, asset_family=1)
		r.set('vcgEditorial_last_last_id', current_last_id)
		r.set(flag_tencent, 2)
		r.set(flag_web, 2)

		time.sleep(1000)
		continue

	elif ( flag_tencent_state == 2 and flag_web_state == 2) or( flag_tencent_state == 1 and flag_web_state == 2)or ( flag_tencent_state == 2 and flag_web_state == 1):
		print("更新未完成，我将在60s后再次请求")
		time.sleep(60)
		continue

	else:
		print('错误停止')
		warning_email()
		break